add kitsune.l10n app for handling content localization

Author: escattone

docker-compose.yml

@@ -67,6 +67,19 @@ services:
       - postgres
       - redis
 
+  beat:
+    build:
+      context: .
+      target: base
+    command: celery -A kitsune beat -l info
+    env_file: .env
+    volumes:
+      - ./:/app:delegated
+    user: ${UID:-kitsune}
+    depends_on:
+      - postgres
+      - redis
+
   mailcatcher:
     image: schickling/mailcatcher
     ports:

kitsune/community/utils.py

@@ -8,6 +8,7 @@
 from django.db.models.functions import Now
 from elasticsearch_dsl import A
 
+from kitsune.l10n.utils import SUMO_L10N_BOT_USERNAME
 from kitsune.products.models import Product
 from kitsune.search.documents import AnswerDocument, ProfileDocument
 from kitsune.users.models import ContributionAreas, User
@@ -145,6 +146,7 @@ def top_contributors_l10n(
 
     users = (
         User.objects.filter(created_revisions__in=revisions, is_active=True)
+        .exclude(username=SUMO_L10N_BOT_USERNAME)
         .annotate(query_count=Count("created_revisions"))
         .order_by(F("query_count").desc(nulls_last=True))
         .select_related("profile")

kitsune/l10n/__init__.py

@@ -0,0 +1,139 @@
+from datetime import timedelta
+
+from django import forms
+from django.conf import settings
+from django.contrib import admin
+from django.core.exceptions import ValidationError
+from django.core.validators import validate_slug
+from django.urls import reverse
+from django.shortcuts import redirect
+
+from kitsune.l10n.models import MachineTranslationSettings
+from kitsune.l10n.utils import duration_string, parse_duration
+
+
+def require_one_of_days_hours_or_minutes(value):
+    if "," in duration_string(value):
+        raise ValidationError(
+            (
+                "Must be expressed as an integer number of only one of the "
+                'following units of measurement: "days", "hours", or "minutes" '
+                '(for example, "3 days" or "2 hours" or "1 minutes").'
+            )
+        )
+
+
+class SimpleDurationField(forms.DurationField):
+    def prepare_value(self, value):
+        if isinstance(value, timedelta):
+            return duration_string(value)
+        return value
+
+    def to_python(self, value):
+        if value in self.empty_values:
+            return None
+        if isinstance(value, timedelta):
+            return value
+        try:
+            value = parse_duration(str(value))
+        except OverflowError:
+            raise ValidationError(
+                self.error_messages["overflow"].format(
+                    min_days=timedelta.min.days, max_days=timedelta.max.days
+                ),
+                code="overflow",
+            )
+        if value is None:
+            raise ValidationError(self.error_messages["invalid"], code="invalid")
+        return value
+
+
+class MultipleSlugField(forms.Field):
+    widget = forms.Textarea(attrs=dict(rows=3, placeholder="Enter each slug on a new line."))
+
+    def prepare_value(self, value):
+        if isinstance(value, list):
+            return "\n".join(value)
+        return value
+
+    def to_python(self, value):
+        if not value:
+            return []
+
+        result, errors = [], []
+        for slug in value.splitlines():
+            if not slug.strip():
+                continue
+            try:
+                validate_slug(slug)
+            except ValidationError:
+                if not errors:
+                    errors.append(
+                        "A valid slug consists of letters, numbers, underscores or hyphens."
+                    )
+                errors.append(f"'{slug}' is not a valid slug.")
+            else:
+                result.append(slug)
+
+        if errors:
+            raise ValidationError(errors)
+        return result
+
+
+class MachineTranslationSettingsForm(forms.ModelForm):
+    LANGUAGE_CHOICES = tuple(
+        (lang, f"{settings.LOCALES[lang].english} ({lang})")
+        for lang in settings.SUMO_LANGUAGES
+        if lang not in ("xx", "en-US")
+    )
+
+    heartbeat_period = SimpleDurationField(
+        validators=[require_one_of_days_hours_or_minutes],
+    )
+    review_grace_period = SimpleDurationField()
+    post_review_grace_period = SimpleDurationField(label="Post-review grace period")
+    enabled_languages = forms.MultipleChoiceField(
+        choices=LANGUAGE_CHOICES,
+        widget=forms.CheckboxSelectMultiple,
+        label="Languages enabled for machine translation",
+        required=False,
+    )
+    restrict_to_slugs = MultipleSlugField(
+        label="Restrict machine translation to these KB article slugs",
+        required=False,
+    )
+    excluded_slugs = MultipleSlugField(
+        label="Disable machine translation for these KB article slugs",
+        required=False,
+    )
+
+    class Meta:
+        model = MachineTranslationSettings
+        fields = "__all__"
+
+
+@admin.register(MachineTranslationSettings)
+class MachineTranslationSettingsAdmin(admin.ModelAdmin):
+    list_display = (
+        "is_enabled",
+        "llm_name",
+        "heartbeat_period",
+        "review_grace_period",
+        "post_review_grace_period",
+        "enabled_languages",
+        "restrict_to_slugs",
+        "excluded_slugs",
+        "restrict_to_approved_after",
+    )
+
+    form = MachineTranslationSettingsForm
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def changelist_view(self, request, extra_context=None):
+        obj = MachineTranslationSettings.load()
+        return redirect(reverse("admin:l10n_machinetranslationsettings_change", args=[obj.id]))

kitsune/l10n/admin.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class L10nConfig(AppConfig):
+    name = "kitsune.l10n"
+    default_auto_field = "django.db.models.BigAutoField"
+
+    def ready(self):
+        import kitsune.l10n.signals  # noqa

kitsune/l10n/apps.py

@@ -0,0 +1,139 @@
+Assume the role of an expert at translating software-related technical documents from {{ source_language }} to {{ target_language }}.
+{%- if example %}
+
+Next, I'm going to provide you with what we'll call the "antecedent" example. The "antecedent" example consists of two parts. The first part will be a piece of {{ source_language }} text, and the second part will be the translation of that {{ source_language }} text into {{ target_language }}. Remember this "antecedent" example, because it will be needed later. The "antecedent" example will be delimited by <<<begin-antecedent-example>>> and <<<end-antecedent-example>>>.
+
+<<<begin-antecedent-example>>>
+Here is the {{ source_language }} text of the "antecedent" example (delimited by <<<begin>>> and <<<end>>>):
+
+<<<begin>>>
+{{ example.source_text|safe }}
+<<<end>>>
+
+Here is the corresponding {{ target_language }} translation of the {{ source_language }} text of the "antecedent" example (delimited by <<<begin>>> and <<<end>>>):
+
+<<<begin>>>
+{{ example.target_text|safe }}
+<<<end>>>
+<<<end-antecedent-example>>>
+{%- endif %}
+{%- if include_wiki_instructions %}
+
+Next, remember the following definitions of new terms (delimited by <<<begin-term-definitions>>> and <<<end-term-definitions>>>):
+
+<<<begin-term-definitions>>>
+Definition of "wiki-hook":
+A "wiki-hook" is a string that case-sensitively matches the entire regular expression that follows (delimited by <<<begin>>> and <<<end>>>, and specified using Python’s regular expression syntax in Python's raw string notation):
+
+<<<begin>>>
+r"\[\[(Image|Video|V|Button|UI|Include|I|Template|T):.*?\]\]"
+<<<end>>>
+
+Definition of "wiki-article-link":
+A "wiki-article-link" is a string that case-sensitively matches the entire regular expression that follows (delimited by <<<begin>>> and <<<end>>>, and specified using Python’s regular expression syntax in Python's raw string notation):
+
+<<<begin>>>
+r"\[\[(?!Image:|Video:|V:|Button:|UI:|Include:|I:|Template:|T:)[^|]+?(?:\|(?P<description>.+?))?\]\]"
+<<<end>>>
+
+Definition of "wiki-external-link":
+A "wiki-external-link" is a string that case-sensitively matches the entire regular expression that follows (delimited by <<<begin>>> and <<<end>>>, and specified using Python’s regular expression syntax in Python's string and raw string notation):
+
+<<<begin>>>
+r"\[((mailto:|git://|irc://|https?://|ftp://|/)[^<>\]\[" + "\x00-\x20\x7f" + r"]*)\s*(?P<description>.*?)\]"
+<<<end>>>
+{%- if example %}
+
+Definition of "antecedent-wiki-map":
+The "antecedent-wiki-map" is a Python dictionary built from the "antecedent" example provided earlier. A Python dictionary maps keys to their values. Each "wiki-hook", "wiki-article-link", and "wiki-external-link" in the English text of the "antecedent" example becomes a key in the "antecedent-wiki-map", and each key's value is its corresponding translation found in the {{ target_language }} text of the "antecedent" example.
+{%- endif %}
+<<<end-term-definitions>>>
+{%- endif %}
+
+Next, remember the following "special instructions" (delimited by <<<begin-special-instructions>>> and <<<end-special-instructions>>>):
+
+<<<begin-special-instructions>>>
+Special Instruction 1:
+The following case-sensitive strings of text, each delimited by quotes, must not be changed:
+- "Anonym"
+- "Bugzilla"
+- "Camino"
+- "Fakespot"
+- "Firebug"
+- "Firefox"
+- "Firefox for Android"
+- "Firefox for iOS"
+- "Firefox for Enterprise"
+- "Firefox Focus"
+- "Firefox Relay"
+- "Firefox Developer Edition"
+- "Firefox Friends"
+- "Firefox Nightly"
+- "Firefox OS"
+- "Firefox Rocket"
+- "Foxkeh"
+- "Lightbeam"
+- "MDN"
+- "MDN Plus"
+- "Minimo"
+- "Mozilla"
+- "Mozillians"
+- "Mozilla Communities"
+- "Mozilla Reps"
+- "Mozilla Webmaker"
+- "Mozilla Wordmark"
+- "Mozilla Wordmark + Symbol"
+- "Mozilla VPN"
+- "Mozilla Monitor"
+- "Pocket"
+- "Pontoon"
+- "QMO"
+- "SUMO"
+- "Sunbird"
+- "Sync"
+- "Thunderbird"
+- "Thunderbird for Android"
+- "View Source"
+- "VPN"
+- "XUL"
+- "Android"
+- "iOS"
+- "Linux"
+- "Mac"
+- "MacOS"
+- "Windows"
+{% if include_wiki_instructions -%}
+- "{note}"
+- "{/note}"
+- "{warning}"
+- "{/warning}"
+- "{/for}"
+- "__TOC__"
+
+Special Instruction 2:
+Strings that case-sensitively match the entire regular expression (specified using Python’s regular expression syntax in raw string notation) r"\{(for|key|filepath) .*?\}" must not be changed.
+
+Special Instruction 3:
+For strings that case-sensitively match the entire regular expression (specified using Python’s regular expression syntax in raw string notation) r"\{(?:button|menu|pref) (?P<description>.*?)\}", only translate the text within the named group "description", and keep the rest of the string unchanged.
+
+{% if example -%}
+Special Instruction 4:
+Each "wiki-hook" must be translated as follows. First, check if the "wiki-hook" is a key within the "antecedent-wiki-map". If it is a key within the "antecedent-wiki-map", use its value from the "antecedent-wiki-map" as its translation. If it is not a key within the "antecedent-wiki-map", it must remain unchanged.
+
+Special Instruction 5:
+Each "wiki-article-link" must be translated as follows. First, check if the "wiki-article-link" is a key within the "antecedent-wiki-map". If it is a key within the "antecedent-wiki-map", use its value from the "antecedent-wiki-map" as its translation. If it is not a key within the "antecedent-wiki-map", only translate the text within its named group "description", and keep the rest of the string unchanged.
+
+Special Instruction 6:
+Each "wiki-external-link" must be translated as follows. First, check if the "wiki-external-link" is a key within the "antecedent-wiki-map". If it is a key within the "antecedent-wiki-map", use its value from the "antecedent-wiki-map" as its translation. If it is not a key within the "antecedent-wiki-map", only translate the text within its named group "description", and keep the rest of the string unchanged.
+{%- else -%}
+Special Instruction 4:
+Each "wiki-hook" must not be changed.
+
+Special Instruction 5:
+For each "wiki-article-link", only translate the text within its named group "description", and keep the rest of the string unchanged.
+
+Special Instruction 6:
+For each "wiki-external-link", only translate the text within its named group "description", and keep the rest of the string unchanged.
+{%- endif %}
+{%- endif %}
+<<<end-special-instructions>>>

kitsune/l10n/jinja2/l10n/llm_system_message.txt

@@ -0,0 +1,31 @@
+Please translate the {{ source_language }} text (delimited by <<<begin>>> and <<<end>>>) into {{ target_language }}, by strictly obeying each of the following steps:
+
+{% if example and include_wiki_instructions -%}
+Step 1: Build the "antecedent-wiki-map", and remember it.
+
+Step 2: Compare the {{ source_language }} text you've been asked to translate with the {{ source_language }} text of the "antecedent" example, and determine which parts are the same and which parts are different.
+
+Step 3: For each part that is the same, copy its corresponding translation from the {{ target_language }} text of the "antecedent" example.
+
+Step 4: For each part that is different, freshly translate that part while obeying the "special instructions" defined earlier.
+
+Step 5: Review your translation, replacing each "wiki-hook", "wiki-article-link", and "wiki-external-link", if it matches a key within the "antecedent-wiki-map", with its value within the "antecedent-wiki-map".
+
+Step 6: In your response, include the "antecedent-wiki-map", describe what you did for each step, and delimit your resulting translation with {{ result_delimiter_begin|safe }} and {{ result_delimiter_end|safe }}.
+{%- elif example -%}
+Step 1: Compare the {{ source_language }} text you've been asked to translate with the {{ source_language }} text of the "antecedent" example, and determine which parts are the same and which parts are different.
+
+Step 2: For each part that is the same, copy its corresponding translation from the {{ target_language }} text of the "antecedent" example.
+
+Step 3: For each part that is different, freshly translate that part while obeying the "special instructions" defined earlier.
+
+Step 4: In your response, include the "antecedent-wiki-map", describe what you did for each step, and delimit your resulting translation with {{ result_delimiter_begin|safe }} and {{ result_delimiter_end|safe }}.
+{%- else -%}
+Step 1: Obey the "special instructions" while translating.
+
+Step 2: In your response, delimit your resulting translation with {{ result_delimiter_begin|safe }} and {{ result_delimiter_end|safe }}.
+{%- endif %}
+
+<<<begin>>>
+{{ source_text|safe }}
+<<<end>>>